from lxml import etree
import requests
class Tieba(object):

    def __init__(self,name):
        self.url = 'https://tieba.baidu.com/f?ie=utf-8&kw={}'.format(name)
        self.headers = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0"}

    def get_data(self,url):
        response = requests.get(url, headers=self.headers)
        with open("temp.html", "wb")as f:
            f.write(response.content)
        return response.content

    def parse_data(self, data):
        #创建element对象
        data = data.decode().replace("<!--","").replace("-->", "")
        html = etree.HTML(data)
        el_list = html.xpath('//li[@class=" j_thread_list clearfix thread_item_box"]/div/div[2]/div[1]/div[1]/a')
        print(len(el_list))
        data_list = []
        for el in el_list:
            temp = {}
            temp['title'] = el.xpath("./text()")[0]
            temp['link'] = 'http://tieba.baidu.com' + el.xpath("./@href")[0]
            data_list.append(temp)
        #获取下一页url
        try:
            next_url = 'https:' + html.xpath('//a[@class="pagination-item"]/@href')[0]
        except:
            next_url = None
            return data_list,next_url
        return data_list

    def save_data(self,data_list):
        for data in data_list:
            print(data)
    def run(self):
    #url
    #headers
    #发送请求，获取响应
        next_url = self.url
        while True:
            data = self.get_data(next_url)
            self.parse_data(data)
        #从响应中提取数据（数据和翻页url）
            data_list,next_url = self.parse_data(data)
            self.save_data(data_list)
            #判断是否end
            if next_url == None:
                break

if __name__ == '__main__':
    tieba = Tieba("itcast")
    tieba.run()
